{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.3 模型评估方法 - ROC曲线与KS曲线"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.案例实战 - 股票客户流失预警模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      " 0 0 0 0 0 1 1 1 0 0 1 0 1 0 0 0 0 1 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 0 0 0\n",
      " 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 1 1 0 0 0 1 0 0 0 1]\n",
      "0.7977288857345636\n",
      "[[0.82041491 0.17958509]\n",
      " [0.84029613 0.15970387]\n",
      " [0.79819342 0.20180658]\n",
      " [0.62989192 0.37010808]\n",
      " [0.61636611 0.38363389]]\n"
     ]
    }
   ],
   "source": [
    "# 1.读取数据\n",
    "import pandas as pd\n",
    "df = pd.read_excel('股票客户流失.xlsx')\n",
    "\n",
    "# 2.划分特征变量和目标变量\n",
    "X = df.drop(columns='是否流失') \n",
    "y = df['是否流失']\n",
    "\n",
    "# 3.划分训练集和测试集\n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)\n",
    "\n",
    "# 4.模型搭建\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "model = LogisticRegression()\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "# 5.模型使用1 - 预测数据结果\n",
    "y_pred = model.predict(X_test)\n",
    "print(y_pred[0:100])  # 打印预测内容的前100个看看\n",
    "\n",
    "# 查看全部的预测准确度\n",
    "from sklearn.metrics import accuracy_score\n",
    "score = accuracy_score(y_pred, y_test)\n",
    "print(score)  # 打印整体的预测准确度\n",
    "\n",
    "# 6.模型使用2 - 预测概率\n",
    "y_pred_proba = model.predict_proba(X_test)  \n",
    "print(y_pred_proba[0:5])  # 打印前5个客户的分类概率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.3 模型评估方法 - ROC曲线与KS曲线"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4.3.1 分类模型的评估方法 - ROC曲线**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "补充知识点：混淆矩阵的Python实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[968  93]\n",
      " [192 156]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "m = confusion_matrix(y_test, y_pred)  # 传入预测值和真实值\n",
    "print(m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0（预测不流失）</th>\n",
       "      <th>1（预测流失）</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0（实际不流失）</th>\n",
       "      <td>968</td>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1（实际流失）</th>\n",
       "      <td>192</td>\n",
       "      <td>156</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          0（预测不流失）  1（预测流失）\n",
       "0（实际不流失）       968       93\n",
       "1（实际流失）        192      156"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame(m, index=['0（实际不流失）', '1（实际流失）'], columns=['0（预测不流失）', '1（预测流失）'])\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.91      0.87      1061\n",
      "           1       0.63      0.45      0.52       348\n",
      "\n",
      "    accuracy                           0.80      1409\n",
      "   macro avg       0.73      0.68      0.70      1409\n",
      "weighted avg       0.78      0.80      0.79      1409\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "print(classification_report(y_test, y_pred))  # 传入预测值和真实值"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**4.3.2 案例实战 - 评估股票客户流失预警模型**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.17958509, 0.15970387, 0.20180658, ..., 0.04220544, 0.09782449,\n",
       "       0.63586739])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred_proba[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.计算ROC曲线需要的假警报率（fpr）、命中率（tpr）及阈值（thres）\n",
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thres = roc_curve(y_test, y_pred_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # 感兴趣的读者可以查看下roc_curve()函数返回的内容\n",
    "# print(roc_curve(y_test, y_pred_proba[:,1]))\n",
    "# type(roc_curve(y_test, y_pred_proba[:,1]))\n",
    "# len(roc_curve(y_test, y_pred_proba[:,1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.867342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.864187</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.857303</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.040230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         阈值      假警报率       命中率\n",
       "0  1.930369  0.000000  0.000000\n",
       "1  0.930369  0.000000  0.002874\n",
       "2  0.867342  0.000000  0.034483\n",
       "3  0.864187  0.001885  0.034483\n",
       "4  0.857303  0.001885  0.040230"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2.查看假警报率（fpr）、命中率（tpr）及阈值（thres）\n",
    "a = pd.DataFrame()  # 创建一个空DataFrame \n",
    "a['阈值'] = list(thres)\n",
    "a['假警报率'] = list(fpr)\n",
    "a['命中率'] = list(tpr)\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAESCAYAAAAVLtXjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAAAgQElEQVR4nO3deXhV5b328e8vIYwJcwBRwiQIToikDsgQB5zq8FZr1VrrqQOvtrXnnE7HVo+1p3aitqet1lacqq9a62xrVdQiFUVGFWWQQZkhzCGBQMbf+8feCQETEsJee+291/25rlzs4Un2vQjcWVl7recxd0dERKIlK+wAIiKSfCp/EZEIUvmLiESQyl9EJIJU/iIiEaTyF0kgM8sNO4NIS6j8RQ6RmX1iZseaWRugzMzaNTFutJmd0uD+WWZ2cfy2/i9KUukfnESKmd1hZjvNbJOZrTOz78Qfv8HMNphZsZl9o8H4kWa20MzWm9lPm/iyFUCpu1cDte5e0cS4nwD5De4fAxSZ2XXA3Ye+dSItp/KXKLrH3XsBpwH/ZWYjgF8C44HRwE/MbHh8T/4Z4IfAAOBMMzu7ka9XG/8AaPSqSTMbB6wH5pjZZfGHdwF7gD/HHx+XgG0TaZE2YQcQCYu7rzSzmcC5wBR3XwpgZq8CnwfmArvd/cX44y8AZwKvmdn7xP7/7AAGAs+ZWSWQbWZvx1/iNKATkAtMAi4Dvg90MLNq4BzgpPhHafz13gp6u0VA5S8RZmYFQCGQB8xq8NRqYnv6e4AlDR5/mFiZA1QB/+nu08xsAXCJu681s2p3HxP/raEq/nEyUAC8EP/804FbgGVAubtfE89zrJn1c/c1QWyvSEM67CNR9E0z20SsfH8FfErsuH2dSqAD0BXYWfegu29090/jd2tpGQdeAg4ntld/E7AZeAKoBsaa2avx30DujI8TCZz2/CWK7gH+B1gD/AMYBLRv8Hw7oJzYXnv9mTtmNh7o5+6PxR+628wOdNgHAHd3M/sKsWL/lNhvFTcAc4jtgH0JuAAoc/eZid1UkcZpz18iyd3LgYeAr7O3kOv0B1YAy4n9YKgzFji+wf2b3X1MfOwl8ds18T+L6gaZWQ6xvfq+wC+AYe5+vbvfB6wEros/vyVxWyhyYCp/ibI/AFcDU4GzzewoMxtM7I3YfwCvAwPN7Mz4xVuXAW8e7Iu4e5W7DyD2ZvFvgCMaPL2UWPGfA2ivX5JGh30ksuJn+7xF7A3Z7wHTiO0Q/dDdlwCY2fnA/UAv4AF3fyX+6UYLD/uY2XBgClAGTAfeMbOBxN5vyAJmEyv/sWZW4O53BLfVIjGmxVxEDp6ZfQD8h7tPa+L5urN9Orl7ebzUV8efG0XsN4vb3X2ymXWL3y8Eitx9RjK2QaJN5S8SAjNr6+6VDe5nAb3cvTjEWBIhKn8RkQjSG74iIhGk8hcRiaC0ONunZ8+ePmDAgLBjiIiklXnz5m1x9/zGnkuL8h8wYABz584NO4aISFoxs1VNPafDPiIiEaTyFxGJIJW/iEgEqfxFRCJI5S8iEkEqfxGRCAqk/M2st5lNP8DzOWb2kpnNMLNrg8ggIiJNS/h5/vEZCh9h71qnjbkZmOvud5jZc2b2tLuXJTqLiEiyLN1Yxkvz1yf86xYO6M64oY1ep3VIgrjIqwa4HHjxAGOKiC1gDTCD2FS2+yySYWYTgYkABQUFCQ8pItKUfy3dzOayigOOeXzWKhZvKCUnO3YApWxPNQBmic1y4/jB6VH+7l4KYAf+G+gErIvfLgV6N/J1JgOTAQoLCzX1qIgclB3lVSzddHAHFLaUVfDfLy5gy87K5gfHXXnS3p3TUf27ccHxfQ/qNcMS1vQOO4EOwA4gN35fROSguDvvLN/KYzNXkbXfO5gvf9T6pRH6dG7PXZeNoH+Pjgccl5/XjvY52a1+nTCFVf7zgDHAM8AItHapiDTj/727knUle/Z57KF3VlBZXQvAwJ6daJO194jDkb1y6d+9I187beBBvU6ndtmc0K9rc0cv0l7g5W9mZwBHu/s9DR5+BHjZzMYCRwOzgs4hIomzo7yKKYuKqakN9ojs+6u389Tctfs81rbN3l38yupaOuRk86erRzE+gOPimSyw8nf3ovifU4Gp+z23yswmENv7v93da4LKISKHbsvOCpYW7z1+fvVDswMv/oYuL+xHz7y2XHPqAHp1bp+0181koU3p7O7rgafCen0RgR27qyjbU9Xoc+7wqylLKK+s4Y3FGxsdM/MHZwYZD4gdhslrnxP460RNWsznLyKH5rWFxcxasW2fx3ZVVPPknDUt+vzhh3VmaO9cvtzgzJaj+uTRtWPbhOaU5FH5i2QId+elDzewvXzvaYo/e3kxe6pq6+/nttv7X766tpYsg0tPPILPDeze6NfMyTYmHN1nn8+TzKDvqEgKq6l1Zq/YRkX1vm+Lrdyyi3unfUKHttlkxc9K2Vi6h/LKz7591jY7i4njBnHusX049vAuScktqU/lL5KCdlZUs7msgl+8spgpCxs/3g4wsqAr/brFzkU/Ll7sN44fTO/O7erHdO/UNuNPW5SDp/IXSTErtuzi9Lum7fPYE9efTPu2+15M1Ll9Dkf2yk1iMskkKn+RFFFT6/xl9mpue2EBAGOH9OTSE4/gqD55DD+sc8jpJNOo/EVSxM1/ea9+SoKLT+jL764YGXIiyWQqf5EkcndWbi2nqmbvGThle6r45StLWLShFIidO9+niy5kkmCp/EWSZElxGdf+eQ7rSnY3+nz7nCwmXXq8il+SQuUvEqAd5VU8PW8NVTXOv5ZuYl3JbtrnZPHT/3PcPrNBdmyXzbgh+WRn6awcSQ6Vv0hA7vjbQv48Y+U+jw3plcvr3x4fTiCRBlT+IgmyZls5P3z+I3ZX1rCtvJJPN+8C4OtFg7n5jCGYUb/qk0jYVP4ih6im1rn9xQU8Pmt1/WOjB/egR6e2/PD84Yws6BZiOpHGqfxFDoG7M/z2V+sXFPneOUfxb6MH0Elz4UiK079QkYNUVVPL3z5Yz3eenr/P43NuPYv8vHZNfJZIalH5ixxA6Z4qlm/ad4npiY/OY8vOCiD2Bu74ofl8/fQj6d5J0xtL+lD5ixzAt//6AW8s3tToc298e7zm1pG0pfIXacTsFduYvmwzizeUMaxPHrecN2yf50cWdKNLB60uJelL5S+RVlvrXPPwbGat2EbbBqdh7qyoBiDL4Kzh/Sk6qldYEUUCofKXSNq6s4Llm3ayels505dtAeDqU/rvM+bM4b0YPbhnGPFEAqfyl0hxd56Zt5bvPfPhPo8/eu1JjBuaH1IqkeRT+UskrNq6iyfnrOHpuWvrz9S54PjD+PJJBXRom82II7qGG1AkyVT+krF2VVTzj482UFVTy63PxxZIqZs37e/fHMNxR2g9W4kulb9kjJLyShasK+Wvc9ewaP0OPonPrVNnUH4npn6nKJxwIilG5S9pq2xPFSXlVQCUlFdx4T1v7/P8+cf1oX2bbL5/7jCyDHrm6upbkToqf0lLuyqqOe6O1z7z+KD8Tvzy0uMZ1LMTPVT2Ik1S+UvaufOlRTzw9goAhvbOZeK4wQDktmvD2Uf3JksLoog0S+UvaePj4lI2lOxh5oqt9O7cjuvGDORrpw3UHPkiraDyl5T19Nw1/G3+egDc4e3lW+qfGzukZ/0ev4gcPJW/pKzn3lvHh2tLGNonD3c47vAuXHxCXwoHdGdgz05hxxNJa4GUv5k9CAwHXnb3Oxt5vhvwOJAHLHT3G4PIIelrxidbWL9jN8f07cJTN54adhyRjJPwg6VmdgmQ7e6jgb5mNqSRYVcDj7n7WCDPzAoTnUPS147dVXz5/lms2lrOEd07hB1HJCMFsedfBDwVvz0VGAMs22/MVuAoM+sK9ANWI5FVW+t8snknNe78c/EmXl1QDMBXT+3Pjy86JuR0IpkpiPLvBKyL3y4FjmxkzNvA54FvAR8D2/cfYGYTgYkABQUFAcSUVHH2b9/6zGpZ44bmM3HcIMx02qZIEIIo/51A3e/quTR+aOlnwI3uXmpm3wa+BkxuOMDdJ9c9VlhY6AHklBSws6K6vvjvvepEDDiyVy5DeueFG0wkwwVR/vOIHeqZCYwAljQypiNwnJnNBE4G3gggh6SYOSu3sbF0zz6PvbeqBIAfnDeM8487LIRUItEURPm/AEw3s77AecAVZnanu9/WYMzPgYeB/sC7wF8CyCEpZOWWXVz2p3ebfL5wQPckphGRhJd//FBOETABmOTuxcD8/cbMBvROXoZ7cvZqZq/YBsBz78feBvreOUdx9tG99xnXuUMOvTu3T3o+kSgL5Dx/d9/O3jN+JIKmL9vMLc99BEC/7h04rEt7xg/N5/qxA2nXJjvkdCKiK3wl4V76cD3/FV8mUcsjiqQmlb8k1Hurt/PNJ94H4LtnD2XsEC2ALpKKVP6SULc8G9vjv/KkAr55RmMXd4tIKlD5S0J8uLaE6x6Zy+ayCjq3b8PPLzku7EgicgAqfzloC9fvYOG60vr7q7eVc8+by+vvP379KWHEEpGDoPKXFtlRXsWiDbHCv+W5D1m1tfwzY249fzg3jBuU7Ggi0goqf/kMd2ft9t38/JXF1NTGZtaYsnDjPmMuPqEv3z93WP39jjnZdOvUNqk5RaT1VP7yGZf96V3mrto7196wPnkc1TuPft07ct2YgQAce3hn8trnhBVRRA6Ryl/qzV25jS82mILht5efwLnH9qF9ji7KEsk0Kn+hptaZt2o7X3/8PQAG53fi2ZtG07WjDuOIZCqVf8RVVtfyvWfm8+IHsYXSvzDycP738hPCDSUigVP5R9yon7xOWUU1EJuK4aSBml1TJApU/hH25pJN9cX/0s1jOPbwLiEnEpFkUflHTEl5JV95cBalu6tZvS12rv7vrxyp4heJGJV/xPz6taUsWFfKKYO6c2JBV75w4hGM16ybIpGj8o+Qv85ZzSsLigH438tP4LAuHZr5DBHJVCr/CJn06hJ2VlTz72cOUfGLRJzKPwNVVtfy9vLNVFbXAlBd6/zguY8o21PNV0/tz39OGBpyQhEJm8o/A72+aCPfeOK9zzzepUMOXxx1RAiJRCTVqPwzzJLiMh59dyUAD//b5+jTJbYwek62MTg/FzMLMZ2IpAqVf4b5+/z1zFqxjeMO78Ipg3rQoa3m5RGRz1L5Z5A9VTX1i6r8/eYxIacRkVSWFXYASZzHZq4CoH2Ovq0icmDa888A7s4vXvmY+976FIBp3z095EQikupU/hlgc1kF9731KZ3bt+HqU/vXv8krItIUlX+aW7qxjHeWbwHgv84bxlUn9w85kYikA5V/GltXsptvPP4eyzbtBKCH1tAVkRZS+aepJ2ev5pbnPgLg3GP68KOLjtaUDSLSYir/NPXg2ysAuPa0gXzrzCO15KKIHBSVfxr6/T+XsWzTTs45pje3X3h02HFEJA2p/NPIO8u3cOvzH7Fya2wRlmtOHRBuIBFJW4FcDWRmD5rZDDO7rZlx95rZhUFkyDRzV27jqgdmsXJrOaMH9+CJG05m9JE9w44lImkq4Xv+ZnYJkO3uo+PlPsTdlzUybizQx93/nugMmWTZxjKemruGGZ9sBeDKk/rx80uODzmViKS7IA77FAFPxW9PBcYA+5S/meUA9wMvm9nF7v7i/l/EzCYCEwEKCgoCiJn6fv7KYu77V+yq3U5tsxlzZE8Vv4gkRBDl3wlYF79dChzZyJivAouAScDNZlbg7nc3HODuk4HJAIWFhR5AzpR3378+JT+vHXdceAyfP/6wsOOISAYJ4pj/TqDuhPPcJl5jJDDZ3YuBxwBNRhNXW+vcM3UZ1zw0G4B+3Tqo+EUk4YLY859H7FDPTGAEsKSRMcuBQfHbhcCqAHKknbeXbeHqh2bh8d9zRhZ05fvnDgs3lIhkpCDK/wVgupn1Bc4DrjCzO9294Zk/DwIPmdkVQA7wxQBypJU3Fm3k+kfnAtCtYw7P3jSaQfm5IacSkUyV8PJ391IzKwImAJPih3bm7zemDLgs0a+dzu6fHntj9+4rR3LhiL4hpxGRTBfIRV7uvp29Z/xIC8xasQ1AxS8iSaEln1LAxtI9AJw8sHvISUQkKlT+IdtdWVO//KLO6hGRZNHcPiH79WtLeODtFbTJMo7p2yXsOCISESr/kLyxaCM/+ttC1pXsBuD1b49nYM9OIacSkahQ+YfkjcUb2VS2h8L+3bjlvGEqfhFJKpV/CGprneffX0eXDm155qbRYccRkQhS+SeRuzNt6WaWFpdRUV1Lh9rasCOJSESp/JOkttYZO+nN+mP8AH/6yqgQE4lIlB2w/M0sGzgLqHT3N+OPGXCpuz+ThHwZ47VFxfXF/5cbTqF/j4707aoF10UkHM3t+T8B7AJyzewLwCfA9cA/AZV/C9XUOjc+9h4AU/5jHEf1yQs5kYhEXXPl3y++IpcBK4B7gbHuXhJ4sgxRUV3DIzNWAtC1Yw5De2uyNhEJX3Pl397MTiV2JfA24G3gaDPD3WcEni4DvPnxZn728scAPHjN54j9HBURCVdz5T8fuKGR2w6o/FugsiZ2Rs+zN53KqP7dQk4jIhLTXPn/ELgZKAd+F5+KWVqhS4e2YUcQEanX3MRujwILgRJix/vlIGzfVcm3/vI+AFk62iMiKaS5Pf+27v44gJlFfrWtg1FT64z8yev19wf00PQNIpI6miv/fDP7MmBAr/htANz9iUCTpbn5a0sA6JCTzQc/mkCWdv1FJIU0V/6VwJD47b82uC1N2Larkvunf8ofp30CwO+uOIF2bbJDTiUisq/myn+bu/84KUkyxKRXP+bJOWtok2Wcc2wfzj6mT9iRREQ+o7nyP8XMlu73mAHu7kMDypS27pqyhCfnrAFg7m1n0bWjzvARkdTUXPnPcvfTk5IkA/xr6WYgNnePil9EUllzp3pq/p4W2lxWwUfrdnD6UfmcOrhH2HFERA7ogOXv7n9IVpB0d8uzHwJQ0L1jyElERJrX3J6/tIC7s3hDKQB3XHRMyGlERJqn8k+ARRtKWb9jD73y2mniNhFJCyr/BFiwbgcA/33B0SEnERFpGZV/Arz04QYATujXNdwgIiItpPI/RJXVtUxftgWAfnqzV0TShMr/ED0xaxUAF43oG3ISEZGWU/kfonc/3QrAzWccGXISEZGWa+4K31YxsweB4cDL7n7nAcb1Bl5195FB5AhKZXUtj767kh27q5iycCPD+uQxpLcWZReR9JHw8jezS4Ds+MLv95rZEHdf1sTwu4AOic4QtHunLee3b8Q2qU2W8X/HDwo5kYjIwQliz78IeCp+eyowBvhM+ZvZGcAuoDiADIF68YP1ACz48Tl0zMnWXP0iknaCOObfCVgXv10K9N5/gJm1BW4Hbmnqi5jZRDOba2ZzN2/eHEDM1qmtdcorqwHIbddGxS8iaSmI8t/J3kM5uU28xi3AH9y9pKkv4u6T3b3Q3Qvz8/MTn7KVzv/9dDaWVvD54w8LO4qISKsFUf7ziB3qARgBrGxkzFnAN8xsGnCCmT0QQI6Ee/79tXxcXAbAd88+KuQ0IiKtF8Qx/xeA6WbWFzgPuMLM7nT32+oGuPu4uttmNs3drw8gR8J97+nYzJ1TvzOegT21ILuIpK+El7+7l5pZETABmOTuxcD8A4wvSnSGIGzfVUl1rQMwKD835DQiIocmkPP83X07e8/4yQi7q2oAuPX84SEnERE5dIGUf6b5aO0O3loWO+Oocwf9lYlI+lOTNePRd1dy+4sL6+/3ymsfYhoRkcRQ+R+Au/PErNUA/OqLxzPh6N5amF1EMoLK/wDunfYJHxeXMbKgK5cV9gs7johIwmhWzyasL9nNr6YsAWDSpceHnEZEJLFU/k0YN+lNAC4Zebhm7BSRjKPyb8RdU5ZQXetkGfz6SyPCjiMiknAq//18uLaEe95cDsATN5yCmSZuE5HMo/JvYElxGRfd8w4Av79yJKcM6hFyIhGRYKj8G/i4uBSAr5xSoDV5RSSjqfwb+PcnPwDga6cNDDeIiEjAVP5x76/eXn97sCZuE5EMp/KPe291CQB/vOrEcIOIiCSByj9uyoLYUsKjB/cMOYmISPBU/sTm8Jm9chsAXTrmhJxGRCR4Kn9g7qrY8f5hfXQlr4hEg8of+HDtDgDuuOiYkJOIiCSHyh94dcEGAAb00Lq8IhINKn/g0827AOjTRQu1iEg0qPyB9jnZnHNM77BjiIgkjcofyMqCTm21ro2IRIfKX0QkglT+IiIRpPIHyitqwo4gIpJUkS//LTsr2LqrkvJK/QAQkeiIfPm/vmgjACMLuoYbREQkiSJf/s+/vw6Azx9/WMhJRESSJ/LlP3tFbEK3w7t2CDmJiEjyRLr8q2tqATj28M5aqF1EIiWy5b99VyWj7nwDgPOO1SEfEYmWSJa/u3PpH2ewY3cVF47oy6UnHhF2JBGRpAqk/M3sQTObYWa3NfF8FzN7xcxeN7PnzaxtEDmaMvXjTXy6JTaZ2+0XHK0J3UQkchJe/mZ2CZDt7qOBvmY2pJFhVwG/cfcJQDFwbqJzHMgbizcB8OxNo8nPa5fMlxYRSQlBzGZWBDwVvz0VGAMsazjA3e9tcDcf2LT/FzGzicBEgIKCgoQGNIOeue0Y1b9bQr+uiEi6COKwTydgXfx2KdDkXMlmdirQzd1n7v+cu09290J3L8zPzw8gpohIdAVR/juBupPmc5t6DTPrDtwNXBtAhiaVV1bzxKzV1NTWJvNlRURSShDlP4/YoR6AEcDK/QfE3+B9CviBu68KIEOTHn4nFmdQfm4yX1ZEJKUEUf4vAFeb2W+ALwELzezO/cZcB4wCbjWzaWZ2eQA5Gg8Xn87hkWtPStZLioiknIS/4evupWZWBEwAJrl7MTB/vzF/BP6Y6NduzrKNZSzbtJPsLCO3nVbuEpHoCqQB3X07e8/4SRmfxBdq/87ZQ0NOIiISrohd4esAFA3tFXIOEZFwRar8//FRMQA52ZrETUSiLVLln9suG4Aje+lMHxGJtkiVP0CvvHaavllEIi9y5S8iIhEr/1mfbqPWPewYIiKhi9TJ7iW7q9i2qzLsGCIioYvMnn/Znljxf6lQC7eIiESm/NeV7AagS4eckJOIiIQvMuVf58QCzeEvIhKZ8r9ryhIAsrJ0mqeISGTKv2xPNQCnHdkz5CQiIuGLTPkDnDywu2bzFBEhQuWvs/tFRPaKRPnvrKhm9optVNVo6UYREYhI+b+9bAsAR3TrGHISEZHUEInyr9vj//rpg0NOIiKSGiJR/v/4cAMA7dtkh5xERCQ1RKL8667qHdCzU8hJRERSQ8aXf0l5JX+du4aeuW3DjiIikjIyvvzXl+wBYFR/TesgIlIn48u/zhdGajZPEZE6kSl/ERHZK+PLf87KbWFHEBFJORlf/i98sA6AE/t3DTeIiEgKyejy31S2h/dXlwCQn9su3DAiIikko8t/zortAFzxuX6YaR5/EZE6GV3+da4dMzDsCCIiKSUS5S8iIvtS+YuIRFAg5W9mD5rZDDO77VDGiIhIMBJe/mZ2CZDt7qOBvmY2pDVjREQkOEHs+RcBT8VvTwXGtHKMiIgEJIjy7wSsi98uBXq3ZoyZTTSzuWY2d/Pmza0K0qdLe84/ro8WbRcR2U8QrbgT6BC/nUvjP2CaHePuk4HJAIWFha1af31U/26M6j+qNZ8qIpLRgtjzn8fewzgjgJWtHCMiIgEJYs//BWC6mfUFzgOuMLM73f22A4w5JYAcIiLShITv+bt7KbE3dGcCp7v7/P2Kv7ExOxKdQ0REmhbIO6Huvp29Z/O0eoyIiARDV/iKiESQyl9EJIJU/iIiEaTyFxGJIHNv1fVTSWVmm4FVrfz0nsCWBMZJB9rmaNA2R8OhbHN/d89v7Im0KP9DYWZz3b0w7BzJpG2OBm1zNAS1zTrsIyISQSp/EZEIikL5Tw47QAi0zdGgbY6GQLY544/5i4jIZ0Vhz19ERPaj8hcRiaCMKf8oLhrf3PaYWRcze8XMXjez582sbbIzJlpLv4dm1tvM3k9WriAdxDbfa2YXJitXkFrwb7ubmb1sZtPN7E/JzheE+L/Z6Qd4PsfMXor/vVx7qK+XEeUfxUXjW7g9VwG/cfcJQDFwbjIzJtpBfg/vYu9qcWmrpdtsZmOBPu7+96QGDEALt/lq4DF3HwvkmVlan/tvZt2AR4gtcduUm4G58b+XC8ws71BeMyPKn2guGl9EM9vj7ve6++vxu/nApuREC0wRLfgemtkZwC5iP/DSXRHNbLOZ5QD3AyvN7OLkRQtMEc1/n7cCR5lZV6AfsDopyYJTA1xObE3zphSx9+9lBnBIP/AypfwTsmh8mmnx9pjZqUA3d5+ZjGABanab44e2bgduSWKuILXk+/xVYBEwCTjJzG5OUragtGSb3waGAN8CPga2JydaMNy9tAWLWiW0wzKl/BOyaHyaadH2mFl34G7gkI8RpoCWbPMtwB/cvSRZoQLWkm0eCUx292LgMeD0JGULSku2+WfAje7+P8TK/2tJyhamhHZYuhdgnSguGt/s9sT3gp8CfuDurZ0YL5W05Ht4FvANM5sGnGBmDyQnWmBass3LgUHx24W0fhLEVNGSbe4IHGdm2cDJQBQuWEpsh7l72n8AnYH5wG+AxfG/mDubGdMl7NxJ2OabiP06PC3+cXnYuYPe5v3GTws7c5K+z3nA08BbwLvA4WHnTsI2nwQsJLY3/DqQG3buBG37tPifZwDf3O+5/vFt/h0wh9ib4q1+rYy5wjf+bvkE4C2P/frbqjHpJNO2pyW0zdrmKDOzvsT2/qd48+8RHPhrZUr5i4hIy2XKMX8RETkIKn8RkQhqE3YAkXRgZncQuwhnY/yhY4HN8Y8qYlec/ozYm5N7gLXAl929KulhRVpAe/4iLfdTdy9y9yLgnvj9ccDDxC69B7jZ3U8ldhbKWeHEFGmeyl/k0HUDdtfdMTMjdhFOZWiJRJqh8hdpuVvNbJqZ3dvg/lvAKcTOvYbY1dQriR0empr8iCIto2P+Ii33U3d/DOrfA6i/H38MYod/xgAVrvOoJYVpz18k8e4DrotPPSCSklT+Ignm7tuJHfK5NOwsIk3RFb4iIhGkPX8RkQhS+YuIRJDKX0QkglT+IiIRpPIXEYkglb+ISASp/EVEIuj/A+05KVsXSuoLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 3.绘制ROC曲线\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['font.sans-serif'] = ['SimHei']  # 设置中文\n",
    "plt.plot(fpr, tpr)  # 通过plot()函数绘制折线图\n",
    "plt.title('ROC曲线')  # 添加标题，注意如果要写中文，需要在之前添加一行代码：plt.rcParams['font.sans-serif'] = ['SimHei']\n",
    "plt.xlabel('FPR')  # 添加X轴标签\n",
    "plt.ylabel('TPR')  # 添加Y轴标\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8103854528908967"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 4.求出模型的AUC值\n",
    "from sklearn.metrics import roc_auc_score\n",
    "score = roc_auc_score(y_test, y_pred_proba[:,1])\n",
    "score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**补充知识点：对阈值取值的理解**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9303686064502279"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(y_pred_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>分类为0概率</th>\n",
       "      <th>分类为1概率</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>0.069631</td>\n",
       "      <td>0.930369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>0.085373</td>\n",
       "      <td>0.914627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>662</th>\n",
       "      <td>0.092923</td>\n",
       "      <td>0.907077</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1154</th>\n",
       "      <td>0.105118</td>\n",
       "      <td>0.894882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1036</th>\n",
       "      <td>0.105906</td>\n",
       "      <td>0.894094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1093</th>\n",
       "      <td>0.111303</td>\n",
       "      <td>0.888697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1164</th>\n",
       "      <td>0.115550</td>\n",
       "      <td>0.884450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>891</th>\n",
       "      <td>0.116594</td>\n",
       "      <td>0.883406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>437</th>\n",
       "      <td>0.123060</td>\n",
       "      <td>0.876940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1153</th>\n",
       "      <td>0.127293</td>\n",
       "      <td>0.872707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749</th>\n",
       "      <td>0.129633</td>\n",
       "      <td>0.870367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.132658</td>\n",
       "      <td>0.867342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>681</th>\n",
       "      <td>0.133410</td>\n",
       "      <td>0.866590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1327</th>\n",
       "      <td>0.135813</td>\n",
       "      <td>0.864187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>0.136599</td>\n",
       "      <td>0.863401</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        分类为0概率    分类为1概率\n",
       "326   0.069631  0.930369\n",
       "366   0.085373  0.914627\n",
       "662   0.092923  0.907077\n",
       "1154  0.105118  0.894882\n",
       "1036  0.105906  0.894094\n",
       "1093  0.111303  0.888697\n",
       "1164  0.115550  0.884450\n",
       "891   0.116594  0.883406\n",
       "437   0.123060  0.876940\n",
       "1153  0.127293  0.872707\n",
       "749   0.129633  0.870367\n",
       "49    0.132658  0.867342\n",
       "681   0.133410  0.866590\n",
       "1327  0.135813  0.864187\n",
       "264   0.136599  0.863401"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame(y_pred_proba, columns=['分类为0概率', '分类为1概率'])\n",
    "a = a.sort_values('分类为1概率', ascending=False)\n",
    "a.head(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 补充知识点：KS曲线绘制"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "fpr, tpr, thres = roc_curve(y_test, y_pred_proba[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.867342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.864187</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.857303</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.040230</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         阈值      假警报率       命中率\n",
       "0  1.930369  0.000000  0.000000\n",
       "1  0.930369  0.000000  0.002874\n",
       "2  0.867342  0.000000  0.034483\n",
       "3  0.864187  0.001885  0.034483\n",
       "4  0.857303  0.001885  0.040230"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.DataFrame()  # 创建一个空DataFrame \n",
    "a['阈值'] = list(thres)\n",
    "a['假警报率'] = list(fpr)\n",
    "a['命中率'] = list(tpr)\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAECCAYAAAAMxDf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8+yak3AAAACXBIWXMAAAsTAAALEwEAmpwYAABB3UlEQVR4nO3dd3gVZdrH8e+TXkkhIQGSUEPvBAQkEJoURUERK6ioWFDXLSq6LKIvK0VXXayLiiiiiNIEQXqVKoRIbyFAAoGEhFRSz/P+MSG0AIGcnDlJ7s91cXEyZ87MnTH8nDzzFKW1RgghRMXmYHYBQgghyk7CXAghKgEJcyGEqAQkzIUQohKQMBdCiErAyYyTBgQE6Lp165pxaiGEqLC2b9+erLUOLOk9U8K8bt26/PHHH2acWgghKiyl1LFrvSfNLEIIUQlImAshRCUgYS6EEJWAKW3mJcnPzyc+Pp6cnByzS7Erbm5uhISE4OzsbHYpQgg7ZjdhHh8fj7e3N3Xr1kUpZXY5dkFrzdmzZ4mPj6devXpmlyOEsGN208ySk5ND9erVJcgvoZSievXq8tuKEOKGShXmSqkgpdT667zvrJRapJTaqJQacavFSJBfTa6JEKI0btjMopTyA74BPK+z24vAH1rrcUqpuUqpn7TWGdYqUgghKqqNh5PZHHsWgE4nvsArvCutug2y+nlK02ZeCDwALLjOPlHA6KLXG4EIYPWlOyilRgIjAcLCwm62TpvYuXMnAG3atDG1DiFExbbxSDInz+Xw+dojHD6TCYCXOs/LLl/yuwbMCHOtdTrc8Nd9TyCh6HU6EFTCcaYCUwEiIiLsckUMCXMhxK2IT83mVJrxbGtbXAqTfztw2ftznutMe8se+EYT2b1fudRgrd4smYA7kAZ4FX19y95auIe9J9OtUVexZrWq8ebA5td8//XXX2fevHkAzJgxg8jISLZs2UJ2djaBgYHMmjULJycnoqKiuOeee/j666/5888/rVqjEKJi+XZTHJuOnGXJ7sSr3vtyeASNg70J9nHD2dEBNhRNYVK7XbnUYq0w3w50BX4GWgObrXRcm5kwYQKNGzcG4PHHH2fcuHFERkbyxhtvMGrUKBYsWMB9993HqVOnUEpJkAtRha07mMS86ATmRRsNEg0CPenbPJguDQIACPFzp27AFY8ZE7aDXz3w8C+Xmm46zJVSPYFmWuuPL9n8DbBYKRUJNAO2lKWo691B21L79u0BaNWqFXFxcQD4+Pjw0ksvmViVEMIWEtNyWHPgDFe2CU9dF8vR5Kzir2c82ZHI8BInMrxcwg4I62zdIi9R6jDXWkcV/b0KWHXFe8eUUn0w7s7Haq0LrVmkrbi7u3P2rPHUWWvN1q1b6du3L9HR0fTv3x8ADw8PHBzspnu+EKIcZOUW8Ni0rRw4XXKnvMZB3sx5vgterqWM0PRTkJ4AtdtbscrLWW0EqNb6JDDbWsczQ58+fRg6dCgzZ87E1dUVLy8voqKiCA4OZuDAgWaXJ4QoR+k5+aSfz+d4SjYPf2E0LjQJ9mb6Ex0v208pqOHtenNjQE7uMP6uCGFeGfj7+7NixQoAxo0bR1RUFFFRUZfts2bNGtsXJoQoN+fzCvnPsgN8ueHoZds71PXjk4fbUaOaW9lPkrAdlCPUbFX2Y12DhPk1jBs3zuwShBDlbPne0zz97cWFch7sEEq7On4EV3OjW6NStIOXVsJ2CGoOzu7WO+YVJMyFEFVGfGo2h89ksvVoCvOjEzhZ1De8cZA3v77UFSfHcngeZrFAQjS0uNf6x76EhLkQokootGgGffI7yZl5xdvubVubUT0bUj/As/zmQUo5Arlp5dpeDhLmQogq4sMVB0nOzKN/i2Ce7lafMH8PArxcy//ECduNvyXMhRCibFKz8vho1WH8PV14+54WBHrbIMQvSNgOLl4Q2LhcTyMdpq+QlpZGz549iYqKKh7eL4So2J6faXQN/FufRrYNcoBjm6BmG3BwLNfTSJhfISYmhi5durBmzRoGDx5sdjlCiDLQWhOblMmmoiloH+1Ux7YFnNkHp3dBkzvL/VT22cyyZDQk7rLuMYNbQv+J193lv//9L19//TXnzp1jw4YNNG/enNjYWJlsS4gKasrKw3yw4iAAg9vWtn0BMT+AgxO0vL/cTyV35pf4y1/+wocffsjjjz/OmjVrCAwMJDIykrVr1xIUFMSCBcaU7jLZlhD2b8OhZJbuMWYz/Oihtky4t6VtC7AUwp+zoWEf8LJin/VrsM878xvcQduSTLYlRMWTmVvAo18ZQ/J7NA5kYOtati8idjVknIL+k2xyOrkzv4GtW7cCEB0dTcOGDQGZbEsIe7Zy32lavLkUgPfub81Xj3Uwp5CYWeDmC43KZzGKK0ki3cC2bduIiori3LlzMtmWEHZuW1wKT35jDM/v3bQGg9vWxsHBhEXRc9Jh3yJocR842ab3jH02s5joysm1/va3v8lkW0JUAMfOZnH/55sAGHNnU56KrG9eMXvnQ8F5aPOwzU4pYX4dMtmWEBVDTn4h3d9dA8D/hrWnb/Ngcwva+QNUDy/3UZ+XkmYWIUSFlnDuPMO/Mp5tdaznb36QH14BxzdCm4eMyc9tRO7MhRAVVqFFc8f7a8nKMxY3m/V0J5MrAta/b/wdMcKmp5UwF0JUWGczc8nKK+SBiFBe6h1uzsPOywo6Asd+h15jwd3PpqeWMBdCVDi5BYXcNWUDh85kAtCnWRC1fctv4YdSi55hrCjU2nYPPi+QNvNL7Ny5k507d1rlWIWFhQwcOJAePXrw2WefWeWYQgjD9mOpHDqTSYifO5Pua0nvZkFmlwSFBbDzewi/A6rVtPnp5c78EheCvE2bNmU+VkJCAj4+PixcuLDMxxJCXJRfaClecPm/D7ahfR1/kysqcmgZZJ6GdsNMOb1dhvmkrZPYn7Lfqsds4t+E1zq+ds33X3/99eIpb2fMmEFkZCRbtmy5pUm2fvzxRz744APi4uKIiorio48+Yvv27XzzzTcAuLi4MHv2bHx8fIiKiuKZZ55hwoQJLFu2jOBgk5/EC2HHcvIL6TppNQCD2tSiXZht26WvK3oGeNYw7sxNIM0sRSZMmMDo0aMZPXo0K1euBCjVJFtvvvlm8UCjqKgonn/+eR544AFmzZpFv379WLNmDS1bGhP8hIeHs3r1anr37s2XX35ZfO6YmBhiYmIkyIW4jvjUbHq/v5bkzFz8PJz54IE25bfU283KSISDS43uiI7OppRgl3fm17uDtqXSTLL11ltv3dLxFi1aVLx9zJgx9vNDKYSdyS+0MG9HAq/OMW6gohoHMv2JjiZXdYWYH0AXQtvhppUgd+aXcHd3Jzs7GzAmtbf2JFslHQ/Ay8urLGULUWll5xUwYvq24iB/6+7mTDNr4qxrsRTCjm8hrAsENLzx/uXELu/MzdKnTx+GDh3KzJkzcXV1xcvLi6ioKIKDg60yydaJEyfo0aMHbm5uzJ492woVC1F5JWXkMmDKepIycgFY8bduNKzhbXJVJdi7AFJioecYU8tQWmubnzQiIkL/8ccfl23bt28fTZs2tXkt1zJu3LirJt0qi+nTpwPw+OOP3/Rn7e3aCFHejiRl0us/awH4xx2NuKN5MI2C7DDILRb4rAtoCzy/qdzX+VRKbddaR5T0ntyZX4O1J9m6lRAXoio6mpzFjE3HAPh7n0a80DPc5IquY+88SNoH931V7kF+I3YV5lpreRB4BTN+cxLCDKv3n+HrjXGsO5gEQJi/By/2suMgt1hgxVvG7IjNzV/83W7C3M3NjbNnz1K9enUJ9CJaa86ePYubm5vZpQhR7r7dFMeOY6m0C/Olf4uaDDJjAeabcWgZnDsGd/7H9LtysKMwDwkJIT4+nqSkJLNLsStubm6EhISYXYYQ5Wp3QhqrDyRxb9vavP9AG7PLKZ2NH0G1EGj3mNmVAKUMc6XUV0BTYLHWenwJ7/sBMwFvYI/W+tmbLcTZ2Zl69erd7MeEEBVccmYuD07dDECXhgEmV1NKCdvh2Aa449+mDRK60g07TCul7gUctdZdgFpKqZIasYYB32mtIwFvpVSJT1uFEAKgoNDCwdMZ7DieysNfbCYzt4AFo25nSPsK8Fto6jH4qi+4VoN25g0SulJp7syjgAudolcBXYFDV+xzFmislPIFQoHjVx5EKTUSGAkQFhZ2a9UKISqFZ7/bwYp9p4u/Ht65Dq1Dfc0r6Gasexcs+dDnXXCrZnY1xUoT5p5AQtHrdKCkIU4bgDuBl4D9QOqVO2itpwJTwehnfivFCiEqvj0n01ix7zS1fd0Zc2dT6gd60TjYDvuQlyQvC/6cDaG3QcQTZldzmdKEeSZwYdZ3L0pumnkHeFZrna6U+hvwBEXBLYSo2nYnpBF3Nqv460m/7cfb1YnFf4nEx90+2ptL7eBSKMyFbq+YXclVShPm2zGaVjYDrYEDJezjAbRUSm0GbgNWWK1CIUSFlZGTz10fbbhq+8O3hVW8IAfYO9+Y5rZBT7MruUppwnw+sF4pVQvoDzyolBqvtb50IoIJwNdAHWAT8IO1CxVCVBzfbT7GjmOpLIg5CcC7Q1rR5pI28TrVPU2qrAzysuDgMmj7iF30K7/SDcO8qOkkCugDTNZaJwIxV+yzFWheHgUKISqWtQeTGDN/NwCh/u40DvJmSPuQij8Y8OBSKDhvF6M9S1KqfuZa61Qu9mgRQogSzYuO568/xhDg5cq857sQ6u9hdknWs2ee0cQS1tnsSkpkNyNAhRAV16m08/xl1k52Hj9HoyAvPnu0feUK8txMOLQc2j5ql00sIGEuhCgjrTWfrj7CH3EpDG4bwpg7m+Ln6WJ2WdZ16EITyyCzK7kmCXMhxC3bcCiZ8b/uZX9iBlGNA/nP0NZml1Q+9swHryC7bWIBCXMhRCklZeSy+sAZKBryF3/uPFNWGoPBm9WsxvhBLUysrhzlZhozJLYdZrdNLCBhLoS4gbwCCzHx55i+MY5f/zx11ftzn+9CuzA/EyqzkUNLoSDHbnuxXCBhLoQoUWJaDl+sj2XOjnjOZecDEF7Di+kjOhbv4+3mRDW3Cjj452bsmVfUxNLJ7EquS8JcCHGVtQeTeGzaVgAaB3nTsrYPz0c1pF6AJ8E+VWixlAu9WNoNt+smFpAwF0Jc4nxeIY98uZkdx88B8Pmj7ejXoqa5RZnp4G9GE0uzQWZXckMS5kKIYv9bd6Q4yOc934W2lbktvDT2zgevYLtvYgEJcyFEkbOZuXy4wuidEjP2Dnw8Knlb+I1UoCYWKMVKQ0KIquGBoqXb/u+e5hLkcLGJxc57sVwgYS6EIDuvgMNnMmlYw4uHb6tjdjn2Yfdco4kl1P6bWECaWYSo0iwWzZPfbGP1gSQARkbWx9Ghgs9uaA3ZKcZAodueAYeKcc8rYS5EFfbF+lhWH0giqnEgLWr5MLRDqNkl2Ye98411PlsNNbuSUpMwF6KKmrM9nglLjCXcvhgegbNjxbgDLXeFBbDpEwhqCcGtzK6m1CTMhaiCLBZdvIDEe0NbS5Bf6s9ZcPYwPDATKtCCGhLmQlQBWms2HjlLRk4+WbmFfLTqEOfzC5k8pBV9mwebXZ79KMiFNZOgVltocqfZ1dwUCXMhqoCvNhxl/K/7ir9WCoa0D+GOZkEmVmWHdnwLacdh4AcV6q4cJMyFqPT+t/YIE5bsp2vDAP55Z1MAAr1dCfByNbkyO5OXDeveg7Au0KCX2dXcNAlzISqx5MxcJizZD8CbA5sRHuRtckV2bNuXkJkIQ6ZVuLtykEFDQlRqHxUtHjF/1O0S5NeTfx42fAANekLd282u5pZImAtRSWXmFvDNpmO0C/OlVW0fs8uxb0dWwfkU6PyC2ZXcMmlmEaKSiU/N5uNVh9kUexaAl3qF4yCjOq9v30Jw84V63cyu5JbJnbkQlcyY+buZG52Ai6MD4we1oHujQLNLsm+F+XBgMTQeAI4Vd4IxuTMXohJIycpjzYEzJKbnsOZAEm8MaMLIbg3MLqtiiF0LOWnQ9C6zKykTCXMhKrhCi+aF73ew8YjRrNIg0JPhneuaW1RFsvR1cPeHhr3NrqRMJMyFqMC01tz98Qb2nEyncZA3XwyPoEY1V9yc7X8xBbuQsAOSDxoPPp0qdr97CXMhKrD41PPsOZkOwFePRxDi52FyRRXM1i/A2RO6v2Z2JWUmD0CFqKC2H0slcvJqABa92FWC/GYl7oKY76HlfeBWzexqykzuzIWoQOJTs/nHTzGkZuVz4HQGAIPa1KJ5rYofRja341vj765/NbcOKylVmCulvgKaAou11uOvs9+nwBKt9UIr1SeEKJJfaOGxaVtJOHeeqEY1qBfgye0NqzNMHnbevII82PUzNL8X/OubXY1V3DDMlVL3Ao5a6y5KqU+VUuFa60Ml7BcJBEuQC2FduQWFfLbmCCdSznMkKYvHu9Rl3N3NzS6rYtv0sTHis83DZldiNaW5M48CZhe9XgV0BS4Lc6WUM/AFsFgpdY/WesGVB1FKjQRGAoSFhZWhZCGqhgU7ExgzbzcZuQXF22r6uPF8D+k/XiaZZ2D9f8CvHtTvYXY1VlOaMPcEEopepwMNS9hnOLAXmAy8qJQK01p/dOkOWuupwFSAiIgIfcsVC1EFLNuTyMs/7qRdmB+d61cnwMuFx7rURVXA2fzszpqJUJADj/wMjpXnsWFpvpNMwL3otRcl94BpC0zVWicqpb4D/g18VMJ+QohryC0o5Nc/T/FLzEl2xafROMibmU/dJn3GrSnpIGyfDhEjIKCk+9KKqzRhvh2jaWUz0Bo4UMI+h4ELTxEigGNWqU6IKuJAYgYDpqyn0GL80tqxrj9j7moqQW5tK98CZw+IGm12JVZXmjCfD6xXStUC+gMPKqXGa63HXLLPV8A0pdSDgDMwxOqVClFJzdkez99/igHgtnr+/OuuZrSQKWutL+kg7F8E3UeDZ4DZ1VjdDcNca52ulIoC+gCTtdaJQMwV+2QA95dHgUJUZkkZuby1cA8uTg7MGtmJdmF+ZpdUeW35HBxdocNTZldSLkrV+q+1TuVijxYhhBUUFFoY9tUW0nMKmHhvSwny8nTqT4j+DlrcB16Vc0pgGc4vhEkmLNnP/sQM7m8fwtCIULPLqdwW/wPc/aDP22ZXUm4kzIWwsbTsfL5cH8sPW4/TOtSX/xvUQlYCKk9JB+DEFug8qtLelYPMzSKETRUUWnjhhx2sP5SMi5MDk+5rKT1Wylv0d+DgBK0fNLuSciVhLkQ50Vrz+txdxCZlFW+LTc4kOTOPViE+/PxsF1yc5JfjclWYDzE/QKN+4FXD7GrKlfwkCVFO9p3KYNa2E6Tn5OPooHB0UITX8GZoRAgzn7pNgtwWDi2DrCRoO8zsSsqd3JkLUU4+WXMYBwUzn7qN6l4VexWbCsligd+ngGeNCr8kXGlImAthRQcSM9ifaKz8s3LfaWr7uUuQm2XtRDixGQZ9VqnmYLmWyv8dCmEDh05n8Paivaw/lHzZ9r/2bmRSRVXc+VRYOxmCWkLrh8yuxiYkzIUoowOJGTz61RYKLZrujQLp3yKYjvX8cXZ0IMTP/cYHENa37StAw4DJUEVmmpQwF6IMNh05y0NfbMbFyYFfX+xKeJC32SWJ9JPGfOVN7oI6XcyuxmYkzIW4BSlZeazef4b5O42p/r8YHiFBbg8KcmHRX8FSCH3/bXY1NiVhLsRNOp9XyMNfbGZ/orGg8pD2IXRvVHlHFlYYOWnw3zbGcnD9JoJfXbMrsikJcyFugtaafy3YzYHTGXzycDtahfhQ08fN7LIEwOp3jCDvNAo6PWd2NTYnYS7ETfhx2wl+3h7PS73CubNVTbPLERdkJsHWqdDqQej3jtnVmELCXIhSSMrIZeaWY3y44hCR4QH8pVe42SWJS/3+IWgLdHrW7EpMI2EuxA0kpuUQOXkV+YUaDxdH3h3SGkeZ5dB+5GZC9Axo0AtqtTW7GtNImAtxCa01iek5FBRq8gotTNtwlCW7E3FQipd6NWRkt/p4uco/G7sS84Px8LMSrut5M+SnUogiP2+P56c/TrDlaErxNkcHRdtQX/51VzNah/qaV5wVFFoKcXS4/nS7JzNPsvzYcupWq8uyY8t4pOkjhPuG4+zobKMqb5LWsPkzqN0eQjqYXY2pJMyFAE6kZPOPokWVh3WqQ6sQY0HlNqG+Fb7/eH5hPptObeKVta/wz07/5O4Gdxe/l5mXybAlw4gKjWJX0i62JG657LO/HPmF/nX7M7n7ZFuXXTpH10HKEbjz/Soz0vNaJMxFlZaTX8gL30ezYt9pAL5+ogM9Gleeea8LLAX0n9uf09nG9zdmwxgKLYUMDh/MlB1T+GLXFwAcPnf4ss/V96lPbFosAEvilpCen85HPT6yvzv09f8x/m7Q09w67ICEuajS/rvyECv2neauVjUZ1qkOt9WvbnZJVvWPtf8oDvIPoj7gp4M/MXbjWMZuHHvZfnfWv5NfY3/ludbPMbjhYGp6Gd0u8wrzaP9de35P+J0OMzvQuVZnXu/4OmHVwq55zgJLARO3TqRbSDe6hXQrv28uYQccXQs9xoB/vfI7TwWhtNY2P2lERIT+448/bH5eIQBW7D3N2oNJFGrNj9tOMKRdCJOGtDK7LKv5M+lPPtn5CbW8avHzwZ8BWD5kOcGeweQW5vKXVX/h95O/AzCo4SBe7fAq3i7eaK1RJTRVxKbF8u62d9mQsAEADycPxnQaw8AGA4v3KbAUsOr4KuIz4/lg+wfF259p9QybTm4iKjSKp1o+VeLxL8jOzyYzP5MaHqX8zejHR41mlpd3g1u10n2mglNKbddaR5T4noS5qCq01rz685/8tD0eAH9PF+pU92D64x3x8bCz5oObEJ8Rz7bEbRzPOM4jTR/h+RXPsy9lX/H7/+7678vayQsthSSdTyI2LZZONTvhoEq34tH3+75nf8p+jqUfY8eZHfSv15/z+edp6NeQ/Sn7i8P+gubVm7Pn7J7ir4c2Gsobt72Bo4Mjx9OPE5ceR2TtyOKAH795PItjFzP3nrkEewZfv5iUozClLXT7B/QcU6r6KwMJc1HlpWblMXHJfn784wT3tqvNuLubU83N/gO8pB4oOQU5uDkZUwhk5WfR6ftOxe+5OLiQZ8nj7S5v4+7kTp1qdWhavalVayqwFPDFn1/wacynV703osUI7gu/j1petXBQDkzYMoFZB2Zxb/i9zD00lz51+vDGbW/QY3YPAJ5u+TRPt3qasb+P5be43wDwd/Pn675f8+vRX0nMSuTRpo9e/T2sfBs2fGDclfvUtur3Z88kzEWV99Q321ix7wz3tw9h8pBW1/113wz5lnySs5MJ9gxGo1EoZuydwYc7PqShb0NOZZ3ihTYvMH7LeABaVG9BgS5gf8r+4mO0CWyDj6sPFm3h414fl/qO+1ZFn4nmy11f0jqwNXuS9/BM62doVr3ZNff/Zs83vPfHe9c9pq+rL+dyz121feewnRf/p3buBHzYwlik+eEfy/ItVDgS5qJKm7M9nr//FMMDEaFMvK+l3QX5s8ufLW7DvqCJfxP2p+ynVUArjmUcIy037bL3w7zDOJ5xHAAH5cDvD/6Ol4uXzWq+Vb8c+YV/bvgnADHDY/hg+wdM3zMdgNEdR9MjtAdvb36b3xN+v+qzU/tMpXOtzrBgFER/B4//CnW72rJ800mYiypr5pZjjF2wBz8PF5a+HGl363H+Gvsro9dfe+TijmE7UChSc1JZfHQxn+z8hBn9Z9DIrxGHzh3C3cmdUO9QG1ZcdgdSDlDdvToB7gEAZORlkJqTelkPmW2J2ziZeZKBDQbS+tvWxdvHBUZy39aZ0HEkDHjX5rWbTcJcVEnHz2bT7d3VODsqdvyrD9521ka+Ln4do1aOAmBa32l0CDZGMOYW5jLv0Dx61+ldHHhVWWpOKj8d/ImPoj8C4LGsfJ59YBFe1RuaXJntSZiLKsVi0Xy06jA/7zjBiZTz/PZyJE2Czem6NmPvDHILc2kZ0JLm1Zvj6ezJqhOrmHNwDrFpsSRkJlwW5OLanp/WjvWO+ZdtWz10dZX6H971wlwGDYlKJb/Qwms//8nc6ATahPoyMrK+aUF+PP04k7ddHAavUGguv3l6p+s7EuSlkZHIp8eP8HT9ZmzWmcWbe8zuwa7HdplYmP2QMBeVwpn0HGb/cYIluxPZczKdf9zRiFE9GtrsYefCIwt5Y8MbAPSp04cRLUbwzPJnAJh912xSclL4M/lPdiXtYn3Ceka0GEGrwFb0CO1hk/oqNEuhsa6ncuCL/tPJ86/Hy6tfZn3CegBWHV/F7bVvx9XRvp6H2FqpmlmUUl8BTYHFWuvx19kvCPhNa33dSYWlmUVY07a4FO7/fBMAof7ujO7X1GarAJ09f5ao2VHX3UfuHMvo9//C8rHQ/124bSRgDABbF7+OF1a9AEAN9xqsHLrSzCptokzNLEqpewFHrXUXpdSnSqlwrfWha+z+HuBehlqFuCmn03OKg3zCvS15sEOoTe7G/xfzPxbFLiK3MLd429L7luLm5IajcmTuobn8fPBnRnes2nNsl1l2Cqz7D4T3LQ5yAKUU3UO7E+IVQnxmPGfOnyErPwtPZ08TizVXaZpZooDZRa9XAV2Bq8JcKdUTyAISSzqIUmokMBIgLOzak/QIUVopWXm8t/QAAH/r04iHOtrm52rNiTV8vPNjgjyCOJ19GkflyLZHtl02o+ATLZ7giRZP2KSeSqswHxa8AHkZ0HtcibvMuXsOt31/GwCdvu/E4IaDCfQIpGdYT5pXb27DYs13w2aWoiaWKVrrGKXUHUA7rfXEK/ZxAZYBg4D5Wuuo6x1TmllEWU1ZeYj3lx8E4PEudRl3t23+4Z7IOMEDix4gxCuEGQNmkJCRgL+bP75uvjY5f5WRnwNznoT9iyDiSbjr/WvuqrWm1bdXT5T208CfaOLfpDyrtLmy9mbJ5GLTiRdQ0hjh0cAnWutz9ja6TlQuR5OzeH7mDvadSgfgncEtGdi6fNvHCy2FjN04lri0OP5M/hOA96Pex9XRlfq+9cv13FVSXjZMHwAno6HTKOj77+vurpRi+ZDl9JvTj0JdWLz9/oX3A7DhwQ34uPqUa8n2oDRhvh2jaWUz0Bo4UMI+vYGeSqlRQBul1Jda66esV6ao6k6kZPPUN39wMu08jg6KF3o05OnI+lad7fDD7R9S07Mm1d2rE+gRSE3Pmiw5uoTlx5YTkxRTvN+/Ov2LEO8Qq51XXEJrmP8cnNwJd30A7Z8o1QpCwZ7BRA+LBoxwH7VyFOvi1wHQdVZXFg1eRJ1qdcqzctOVJsznA+uVUrWA/sCDSqnxWuvieSe11sUz0Cul1kiQC2v7ZPVhDpzOoGNdfybe15L6gdaZhyQ9L50Rv43gQGpJ9ygXDW82nP71+qO1pmVgS6ucW5Rg+3TYOx96vwURI27qo5e2CnzS6xNOZZ7ijjl3ADDn4Bz+FvE3KxZqf0rbNdEP6AOs01qX+IDzZkibubgZuQWFdHpnJY2DvZk1snOZj1doKUSjcXJwYtzGccw7PA+LtgDQuWZn7qx/J76uvpzIOEEjv0a0CmxVPOWsKEcHl8Ls4RDWCR6dBw5ln/XxRPoJHvj1ATLyMoq3rbp/FYEegWU+thnKPAJUa53KxR4tQthMcmYuYxfsJjU7n+ejyj4Xx7K4Zfx97d9xcnAi1DuUo2lHeaL5E7zY9kWcHJzsbkbFKiEtHlZPgJ3fgUd1GPS5VYIcILRaKC+3e5n/2/x/xdt6/tSTmOEx5T5FsK1Vru9GVCpT1x0hYvwKFu9KpGENL7o2vPU5OFJyUvh2z7e8uu5VADoGd6Rutbr0DO3Jc22ew9nRWYLcDGePwFd9jSBvPABGbYVq1n2gfenydhf8GvurVc9hD2Q4v7ArZ9JzyMk3mjxm/xFPLR83BrerzUu9wnFwuPWwfX3962w8uZG61eoy886ZVHOpGmtG2rVTf8K398D5FKMfede/lstp3J3c2fLwFpwcnLBoC4/99hhToqfQp06fStV8JmEu7ILWmg+WH2TKqsOXbX8uqgGv9L31vsJaa97a9BYbT24kyCOIL+74QoLcbDnpsHsOLPsXuPvCE4uhhnWXtruSh7NH8et/RPyDEUtH0GFmB7Y+shV3p8oxaF3CXJjiTHoO86ITmLXtBEeTs4q3D2pTi8hw4+GUgwP0aFzKldpLkG/Jp92MdoCxruScu+dUif7Gdu3MfpgxCDJOgVeQsVqQn227DHYI7oCPqw9puWmsPL6Su+rfZdPzlxcJc2Fzs7YeZ8z83RRYNI2DvHmpZ0NQipo+bjwQEXpLzSkWbSErPwtvF2+01ny568vi1eKdHJxYPXR1pXvgVeEkbIfvhoCjM9z/DTTqC87m3BWvGbqGAXMH8PXur+lft/9Vi2ZXRBLmotxl5RZwKi2HH7cdZ39iBntOplNg0fzwdCc61fcv84PH2HOx3LPgHgCm95vO9D3TWXNiDc4OzjTya8RPA3+SIDfb0XXww0NGb5Xh88Hf3JGzTg5O9ArrxXf7vmPsxrH8u+v1R5lWBLLSkCgXWmsKLJrcAgst3lx62Xttw3x5qmt9q0xTm5iVSJ+f+1y1/dnWz/JUy6dQKFwcXcp8HnGLtDYGAi15DfzrwbB5UK2W2VUBl//sRA+LxsnB/u9tZaUhYVMpWXk8OHUTB09fXBHm9obVeahjGJHhgfi4l20Ifl5hHptPbeaXI7+QW2BMQXtv+L30CuvFr7G/0jOsJ33r9i3TOYSVbJxizEVeszUMmw8e/mZXVCzYM5gpPabw0uqXaDujbYWfd17CXFhFXoGF348kk5tv4dtNccSdzealXuG4OCqCqrlxf0TZV5BPzUnl8LnDvLz6ZdLz0ou3D2s2jFc7GP3Hu4V0u9bHhS0V5sPSN2DrVAjrYjStONnfSkCRIZHFr+cdmsfg8MEmVlM2EuaiTLLzCpi19QTTN8ZxPCW7ePvk+1oxtEPZA/yC2QdmF4/ic3FwIdwvnKGNhlLDowZdanWx2nmEFexdAL9PgYQ/oMuL0GscONpn1Dg5ODH7rtkMXTSULYlbJMxF1TRjUxwfrDhESlYe4TW8+M/9rWlasxrebk6E+nvc+AClpLVm1oFZADzR3Fj0wc/Nz2rHF1ZQWAB75sGObyDOWJuTQZ9Dm4fMrasUmlZvSo/QHmxL3IbWusKOBJYwF7fkTHoOb/6yhxA/Dybf14qeTWqUaYRmSf69+d/FIQ7wcruXebLlk1Y9h7CC2DXw/YNQcN7opdL2URjwH3CuOKMrO9XsxOoTq/nrmr/yYY8PzS7nlkiYi5s2PzqBN3/Zg0XD9Cc6lGk62pyCHCzagoezB+vi1zFlxxSqu1engW8DFsYuvGzfQQ0HlbFyYVU5abBqvNEu7hkIXV+GyH/YbZPK9fSp04cJWyew8vhKzhecr5CjQiveVRemmbsjnm83HeN4SjauTg68dXfzWwryAykH+Dzmc7Lys9idvJuM/AyCPIJIyUmhllctUnNSmX1gNrmFubzc7mU6BHfgdPZpqrtXL4fvStyS45thzlOQdsJYQKLvv8Gl4i6mHOgRyPR+03n8t8eZfWA2jzV/zOySbpqEuSiV1Kw8/jbbWG0nMjyAIe1DuKdN7Wvuf/b8WZwcnPBx9WHmvpmczj7N4tjFeDp7kp6XTvL5ZAC6h3SnZUBL4tLjcFSOvNLhFXxcfSi0FHIm+ww1PGpUitF5lYalENa9B2sngm+YMRy/blezq7KK9kHtua3mbUzbPY37G91/2XwuFYGEubiuPSfTWLbnNH/GnwPg68c70KPJ9edLiUuL4+FfHyYjPwNPZ0+y8rMue9/ZwZkOwR34qOdHeDqXfDfn6OBITa/yXdtT3KT0k8bd+LHfoeVQuPM/4Fa5Ji0b1WYUw5cM5/mVz/N1368r1MNQCXNxTSv3nebFH6LJzjMWyX2me/0SgzwzL5O18WvZeHKjMd3oqS04Ojgyqs0odifvZkPCBubcPYdAj0C8nb0psBTIQhAVzbnjMP1OyDoLg/8HrR80u6Jy0bZGWwC2n97OGxveYELkBJMrKj0Zzi+ukpVbwNgFe5izIx53Z0fmPNeFZrWuvgNbeGQh03ZP4/A5Y9paL2ev4iHR73Z/l041O9m0blFOkg4aMx3mZRqjOGu3M7uicnXk3BEGLRgEwPoH1uPr5mtqPZe63nB+CXNxmX2n0nnh+x3EJmcxKqohT3ath5/n1XObnMs5R+SPxui5iKAI+tTpQ/96/fFz86vQfXXFFfJz4LPOxtJuI5ZW+iC/4KeDP/H2prcB7GqYv8zNIm5Ia83MLcd5e9FefNydmfnkbXQpYZm2vMI85hyaw+StkwH4fsD3V61WL0FeSeSfh4UvQ0pslbgjv9SQ8CHFYR6XFkddn7rmFlQKEuaC9Jx8Xp+zi193nSIyPIAPHmhDgNfV82h0+aELGXkZOCgHLNrC822evyrIRSVRmG8s6XZii/Gws0EPsyuyKaUUk7tN5tV1r7Ls2DJGthppdkk3JGFexcWcOMcLP+zg5LkcXuvXhGe61b9sJGdOQQ5nc85y5NwRMvIyABjaaCg9QnvQpbbMiVIpndkP856BUzuhxz+h69/MrsgU/ev15/t937MsTsJc2DGtNV9tOMqk3/ZTw9uN2c90on2dy6cnPZN9hudWPMfB1IMA1PKsxcLBC2V+8MrsxDbjjtzZDYZ+C83uMbsiU/Wt25dJ2yZViKYWCfMqaOORZCb9doCYE+e4o1kQk4e0wtfjYkBbtIWvdn3FlOgpgDHFrK+rL51rdpYgr8xSjsJ39xqrAT25DKpJP//edXozadukCtHUImFehew8cY4x83exOyGdmj5uvDukFUPah1z2wDL5fDKvr3+dzac207V2V+6ocweDGg6Sh5qVXV4WTOsHBbkw9BsJ8iLBnsG0rdGWpXFLJcyF+QotmjHzd/PD1uMEVXPlzYHNeKhjGG7Olw+T33RyE6+vf53M/Eze7Pwm94XfJyFeFeRlw3f3QWYi3P1Rleq1Uhp31LmDSdsmcTTtKPV86pldzjXJKreVnNaacb/s4YetxxkaEcKKv3XnidvrXRbkBZYCpuyYwjPLn8HH1Yfv7/yeIY2GSJBXBTt/gM9vNybOuudTaDfc7IrsTp86xjqhy+KWmVzJ9cmdeSWmtebTNUeYsfkYI7vV540BTa/aJzErkdfWvcaOMzsY3HAwozuOrnATDIlbkJkES14xFpRwdIEhX0GL+8yuyi4FeQbRtkZblh1bxjOtnzG7nGuSMK+k9iem8/KsnexPzODu1rUY3a/JZe/nF+az/Nhy3tn6DvmF+UyMnMid9e80qVphU9HfwZLRkJcBtz0LvcZW6OlrbaFv3b5M3DqR2LRY6vvUN7ucEkkzSyWjteavP+5kwH/Xk5yZxxsDmvDu/a2K+46fyznHvEPzGDBvAK+tf41anrWYPXC2BHlVse1LWDAKqteHp1dB/0kS5KXQO6w3YN9NLaW6M1dKfQU0BRZrrceX8L4PMKvoeJnAA1rrPGsWKm5sd0Ia/1qwm+jj5+jSoDpTHmpbPJIz+XwyPx38iWm7ppFTmEMtz1qMbDWSka1G4upof6umCyvLy4Id38Jvo6FRf6PHipP8dy+tIM8g2tVox7Jjy3i29bNml1OiG4a5UupewFFr3UUp9alSKlxrfeiK3R4B3tdaL1dKfQb0A34ph3pFCbTWzNh8jPGL9uHu4sidrWrywdA2uDgZv3jtObuHBxcZU5Y28W/C0MZDGdxwcPEMh6ISy0yCP6bBuslgKYAmd8GQr8FJxgvcrDvq3mHXTS2l+dccBcwuer0K6ApcFuZa608v+TIQOHPlQZRSI4GRAGFhYbdQqihJ2vl8Xvv5T37bk0iPxoH8Z2gb/ItmOYw9F8uprFM8u8K4k3i53cuMaDFCeqlUBQV5EPMDLHzJ+LpuJLR5BFoOAUdnc2uroHqH9WbS1kksi7PPu/PShLknkFD0Oh1oeK0dlVKdAT+t9eYr39NaTwWmgjEF7s2XKq4UfTyVF3+IJjEthzcGNOGprsa8Kh9Hf8ySo0s4nnG8eN9Pen1Ct5BuJlYryl1hAez4xlgJ6FQMnD0Mjq7Qe5zxoNNBHpGVxYVeLUvjllbYMM8ELixV7cU1HpoqpfyBjwDp32Rl+YUWdhxLJSb+HDEn0kjKzAUNO46nElTNjdnPdqZtqC8z981k0rZJxZ+r7VUbV0dXJnebTGP/xiZ+B6LcnYyGBS/A6d3gEwpeQfDAd9CwNzhXvJXm7dWFppbf4n6jX91+ZpdzmdKE+XaMppXNQGvgwJU7KKVcMJpiXtdaH7NqhVVYVm4BGvho1SH+tzYWgBA/d2r7uqMc4P6IEEb3awqO2by0+iXWnFhDTc+ahHqH8mLbF2lTo42Z5QtbKMiDXbNh8Svg7mdMjtX0bpCmtHLRp04fJm6dyCtrXyHYI9iu/o2VJsznA+uVUrWA/sCDSqnxWusxl+zzJNAe+KdS6p/AZ1rrH61ebRXy1sI9fP17XPHXTg6KzW/0umqe8Z1ndvLKuldIPp/Max1e45Gmj0ibeFWQnwP7FsKKcZAeDzXbwCM/gdf1F9sWZVPDowZN/ZuyL2Ufw5YMY8ewHTg72McziFItG6eU8gP6AOu01ollPaksG1eyrUdTePXnGNJzCkjJymNg61q0qu0DQLs6voQFFrIreRf5lnzCfcNZE7+GKTumUNOzJu91f4/mAc1N/g5EuTvwG+yZa/ydmwaBTSFqNDTqK80pNpJbmMtr615j5fGVTO42mf71+tvs3LIGaAWw/VgKw77aSg1vV7qGBxDk7cbw22tSzdWT7/d/z8StE0v8XJ86fXiry1t4u3jbuGJhc3EbjAmxCnKg1YPQ+kGo1w0cHG/8WWFVFm3hzrl3Ep8Zz6y7ZtG8um1upGQNUDumtWbTkbOMnLGdGtVcGTfEm4bV/fhmzzd0/XFW8RJtAK91eI0WAS1wcXTh8LnDeDp50jOspzSrVHaJu2HzZ7DzO/ANM9rFa7U1u6oqzUE5EBUaxXf7vuPBRQ8SMzwGB2VubyEJc5NN/G0//1sbS1h1d7p23MCLa2cXv9evbj9CvUNJz0snIjjisqfnzao3M6NcYQt5WbDwL8ZiETlpcPYQODjB7X+B7qPBRSZCswd/bf9Xvtv3HQCrjq+id53eptYjYW6ipIxcpv8eR9Oa3nRsv565R2bzUJOHaOjbkLTcNB5v8bjdPFwRNnJyJ8x/Ds7sNb5u2Btuewaa3wue1U0tTVzOxdGFncN2cs+Ce3hnyzt0C+lm6kpcEuYm+nJ9LIUusbiGrWbukQM82vRRXu3wqjSbVEXnU43uhbt+MroYPjoXGvYyuypxA44OjjzZ4knGbhxL9x+7s+nhTabVImFukpSsPGZsPkr1hgs4kn6KR5o+IkFelWgNiX/Cmf0QPcMY9JOXCZF/N5pT3HzMrlCU0l3172LsxrFk5mdyvuA87k7m9CqSMDeBxaL5y6xo8txiyNaneLfbu/SrZ1+jyUQ5iv/D6B8et9742tEVfENh4BSoe7uppYmb5+zozLS+0xixdATLjy3n7gZ3m1KHhLkJlu5JZP2hMwQ3XU9gtbrFy1KJSshigZRY48771E6jTfzEZuOBZp//g7pdoUYzcHYzu1JRBhFBEYR5hzH30FwJ86riSFImL/+4k+CwjWRxgn+2egdH6Sdc+Zw7Dr/+Aw4tA4rGcji6QnAL6DgSur9qtI2LSkEpxeDwwfx3x385ln6MOtXq2LwGCXMbyi0o5KUfotEux8nyXISvq69NR4+JcqY1HNsIWz6D/b8a28LvgKYDoVYbCGwi089WYvc0uIePoz9m3qF5vNz+ZZufX8Lcht5beoC9SXF4NfwYgKl9psoCERWd1sa84ft/heSDxh93P+jyEnR4ymgLF1VCoEcgkbUjWXBkAS+0fcHm/7YlSWzklZ9i+Gn7McJazCW1EMZ1HkfT6k3NLkvcDK0hYYfR6wTg5A7Y/DlkJhpt4N61jIeYLe+XgT1V1ODwwayJX8P6+PX0COth03NLmNvAiZRs5sTsI6TRKlILD/Nu93ftbi5kcQ0FuZBxyhiNuWo8JFwxp1C9bsZEV+0ek8UfBJEhkQC8tPolxnYey/2N7rfZuSXMbWDahqO4h00lzTGJQQ0HSZDbK0shbPoEkvZD6jFIjYP0BIofYHoFw10fQEDRQh9eQRBwzYW3RBXk7ODMgHoDWHx0MW9velvCvDJJz8ln9p7VONRKor5PfV7v+LrZJYmSnIw2+n7HrgGPAAgIh3qR4FcXfELAyQ0a9AQPf5MLFfZuUrdJeLt48+OBH0nMSiTYM9gm55UwL0e5BYU8OX0bFu+1+Dr7MXvgbFwdXW/8QVE+Mk4bw+ZTYiHlCCQdgPSTkJsB8VtBOcId46HLi2ZXKiq4x5o/xo8HfmRR7CKeavmUTc4pYV4O8gst/Lw9no9XHeaMWo5b8H4ebfa8BLmtpZ+CA4uNh5ZJ+4y776LphAHwqG5MKevgBD3+CW0fhWq1zKtXVBqh3qG0q9GOhUcW8mSLJ20yTYeEuRWlZOWxdE8in6w+zMmsOALrrMLNKQaAoY2HmlxdFXBmH6yZYMz/rS1w7pjxt5sP+DeAiBEQ1tlo667RTGYhFOVqYIOBvLXpLdYnrKdbSLdyP5+sNGQl6w8l8fS3W8nVqQTXXUuW8xY8nT3pUqsLI1uNpLF/Y7NLrNzOp8Ln3SA72RicU70B+NWD+lEQ1klW4xE2l56Xzu0/GHPt7Hpsl1WOKSsNlbMVe08z6ueFuNWfhrNDBnkOLjzW9DGebPEkvm6+ZpdXOWUlw555sHsuZCUZbeDaAk/8BnU6m12dEFRzqUawZzCJWYnsPLOTNjXalOv55M78Fm2OPcuE36I5eS6XcwUn8AibhqsTDGk8hOHNhtvsCXaVkJ1izDCYuNvoNpidAsc2GO8FtQT/ulCjudHbJOw2U0sV4lLJ55PpMdsYPGSNu3O5M7eiEynZjF+8k9WJs3ENWIfytOCBA8GeQUzr9yW1vWqbXWLlkpMO0/oaw+QBqjc0VqHv9Dw0u8doQhHCTgW4BxS/1lqX64NQCfNSysot4JPVh5gWMxen6ktwDUyjd9gd1PUJ4+z5s4xqM4ogzyCzy6y4LBaj3bvgPJw9cnGekz3zIfss3D8dQjtBtZpmVyrETRnbeSxvb3qb+Ix4QquV31w9EuY3YLFo5kYnMHHlMrK95+AcfJyGPk34V+cptAtqZ3Z5FVN2ChxZZfzJPAMZiXD2sBHkl3LxgjpdjNV35A5cVFAtA1oCsCt5l4S5rVksmpj4c/y25wQLjywjVW3CKegg/i7+/D3ibe5peA8OSubhKJXMM3B8kzFQJ+M0xG+DhO2ANmYX9Ktr3G3X7w4+oeDkCv71IKAReNcEWUZPVHANfBvg5ujGruRdDKg/oNzOI2F+if2n0pm4ZjExp/dx3vEQTl4HUN55OAFPNB/ByFZP4+XiZXaZ9sNSePGuOvmgccednWw0ixTmG/2+zx66uL+zB9RoCt1fg/A+UKutdBkUlZ6zgzNNqzdld/Lucj1PlQ7zQ8mn+Gbnb+w4vZPC3ABO5G3E0T0e/MDfyZ9eYXfTskZTImtHUtOrCrbVFhYYXf5y0owJp9JPXvyTEgunYiiehOoCNx9jZKWDE/jXN0ZV1rkdajQBV29Tvg0hzNYioAWzD8wm35KPs0P5LFBSpcLcYrGw5NAOft67jF2pm8lxiEMpDRZncMzH1yuIx1v8na6hHWni36TiN6VcWAE+Lb5oJsA4yM2E1KOQf9742lJw7c9bCrkqrJ3cjWYR3zCIeMJoGglsYoS1T6ispCNECVoFtGLG3hkcTj1cbusYVPowP5udwYydK1ket5oTOdvRjmkAuFKH9tUe4L4mfRjQuD0JWfHU8qpVbv/XtKmsZIieYTRz/Pnjxe3Onsads28oeNaA2hHgfb0eOMoYSekRYMxZUq2W0c4t7dhC3JQWAS0A4yGohPlN+CP+MN/tWsq207+TpvehHArQFlcCHFvSpVZXhrfpS5PAkMs+Y8YCrKWmtTHDX2EeoCEzCc7sMdqrr5R+EvbOv/h1qwegzcNGk0e1EFlAQQgT1PaqjZ+rH7uTd5fbPE2VIsxz8vOYs3cjvxxcwcH0bRQ4nQTAoTCAcM8+DGjQi4dadcfL1c3kSosU5EHaCaMdWmvjdU5a0Xs5xkjHs4eMVW5y0iDzdMnHcfGCK5uCHF2g9cPQ/jEI6SjhLYQdUErRIqAFu5KtM0dLSSpsmB8/l8T06KWsi1/H6fwYcMxGawe8HMLp5D+Ch1rcQdc6TXGwZZhZCuHccS5rZ04+bHTFy0mDwyuMtuqsJCjMvfZxfMKMNmhnD6NZxM0HvIPBt+i3B3c/o1eILJQgRIXRMqAlGxI2kJWfhaezp9WPX6owV0p9BTQFFmutx9/qPmUVffIon/3xMzuTN5HtcNh4eFnoSU2XdvQIjeKxtn2oVe0WAk5ro2tdXqYRtimxxja4pHmjpM9ZjHbp07uNILfkX3tf5WBMuxp6Gzi5QGBTY8kxRxej98eFebSVI7hK90chKpsWAS3QaPae3UuH4A5WP/4Nw1wpdS/gqLXuopT6VCkVrrU+dLP7WMOW7d+yKfVn6hdCl/OKHhZn2jlYcDq/A9J2wO73b+3AuRlFaz1eh/s1/ifhX89ol3ZyNR4M+jcw5g65wNEFGvQw7qaFEFXWpQ9BTQlzIAqYXfR6FdAVuDKob7iPUmokMBIgLCzslop9qEl3+m3ZTV0f9xvvfDMcnIxV1r2L+pL7hoFrNeO1UsZCvtL2LIQoAz83PwbUG0ANjxrlcvzShLkncOG2NR0oaTnyG+6jtZ4KTAVjCtybrhTwCY/CJzzqVj4qhBCmm9RtUrkduzS3m5nAhVthr2t8pjT7CCGEKCelCd3tGM0mAK2BuFvcRwghRDkpTTPLfGC9UqoW0B94UCk1Xms95jr7yHylQghhQze8M9dap2M84NwM9NBax1wR5CXtk2b9UoUQQlxLqfqZa61Tudhb5Zb3EUIIUT7kQaUQQlQCEuZCCFEJSJgLIUQloLS+pfE7ZTupUknAMZuf2BwBQLLZRdgJuRYXybW4nFyPi653LeporQNLesOUMK9KlFJ/aK0jzK7DHsi1uEiuxeXkelx0q9dCmlmEEKISkDAXQohKQMK8/E01uwA7ItfiIrkWl5PrcdEtXQtpMxdCiEpA7syFEKISkDAXQohKQMLcSpRSXymlNiqlxlzjfT+l1GKl1Hql1Oe2rs+WbnQtLtnvU6XUQFvVZYabuBZBSqloW9VlhlL8G/FRSi1RSi1XSs1TSrnYukZbKs3PRml/fkDC3CouXQMVqKWUCi9ht2HAd1rrSMBbKVUp+9SW8lqglIoEgrXWC21aoA2V9loUeY+LC7xUOqW8Fo8A72ut+wCJQD9b1mhLpbkeN/nzI2FuJVFcvQbqlc4CjZVSvkAocNwmldleFDe4FkopZ+ALIE4pdY/tSrO5KG78c4FSqieQhRFglVUUN7gWWutPtdbLi74MBM7YpjRTRHHjn43S7FNMwtw6rlwDNaiEfTYA4cBLwH4g1Tal2VxprsVwYC8wGeiolHrRRrXZ2g2vRVFTwlhgtA3rMkNpfi4AUEp1Bvy01pttUZhJSnM9Sn3NQMLcWkqzBuo7wLNa67cxwvwJG9Vma6W5Fm2BqVrrROA7oIeNarO10lyL0cAnWutztirKJKVaJ1gp5Q98BIywUV1msfrayhLm1lGaNVA9gJZKKUfgNqCydvAvzbU4DNQveh1B5Z10rTTXojcwSim1BmijlPrSNqXZ3A2vRdFvKbOB17XWlfVn4gLrr62stZY/ZfwDVANigPeBfUUXfvwV+3QE9mD833Y54GV23SZeC2/gJ2AdsAmobXbdZl2LK/ZfY3bNJv9cPIfR/Lim6M8DZtdt8vW4ch+f6x1TRoBaiVLKD+gDrNNG80GVJdfiIrkWF8m1uFxprsfNXDMJcyGEqASkzVwIISoBCXMhhKgEJMyFEKISkDAXFZZSqo1Sqk3R63FKqSgrHvumjlfUtbDMxxHiVkmYi4qsTdEfIao8CXNRISmlJmCMnhytlFpZtLmPUmqtUmqnUiq4aL81Sql3lVJLi772UEr9rJRap5T6pGibu1JqUdG2uUopp5KOp5RyVUr9ULRt5rVm9SuaIXOFUmo1xvwaQpQ7CXNRIWmtXwcmAhO11r2KNjfUWncHvgd6Fm3rBGzSWvct+noksFtr3Q2oqZRqBTQDLEXbpmIMnS7peE8XfbY7cJBrDzkfCSzSWvcA8q3zHQtxfRLmojL5tujvM8CFu+bdWuu5l+zTGBhc1MZdH6gN7AB2K6WWAQOB7GscrxmwpWjbFqDpNeqoB/xZ9PqPW/1mhLgZEuaiIjuPMecNgMKYRvZKmVd8fQD4UGsdBYzBmIq4NfC71voOwA+ILNr3yuPtwbjTp+jvPdeo6xhG8IO06QsbkTAXFdly4F6l1O9cDOAb+QLor5RaBzwLnMCYwOglpdRGIJhr301/CTQv+mw4MP0657iv6O6/WinrEqJMZDi/EEJUAnJnLoQQlYCEuRBCVAIS5kIIUQlImAshRCUgYS6EEJWAhLkQQlQC/w/Bq1TBUZB22wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(thres[1:], tpr[1:])\n",
    "plt.plot(thres[1:], fpr[1:])\n",
    "plt.plot(thres[1:], tpr[1:] - fpr[1:])\n",
    "plt.xlabel('threshold')\n",
    "plt.legend(['tpr', 'fpr', 'tpr-fpr'])\n",
    "plt.gca().invert_xaxis() \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4744656418256471"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max(tpr - fpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "      <th>TPR-FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.930369</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002874</td>\n",
       "      <td>0.002874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.867342</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.034483</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.864187</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.034483</td>\n",
       "      <td>0.032598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.857303</td>\n",
       "      <td>0.001885</td>\n",
       "      <td>0.040230</td>\n",
       "      <td>0.038345</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         阈值      假警报率       命中率   TPR-FPR\n",
       "0  1.930369  0.000000  0.000000  0.000000\n",
       "1  0.930369  0.000000  0.002874  0.002874\n",
       "2  0.867342  0.000000  0.034483  0.034483\n",
       "3  0.864187  0.001885  0.034483  0.032598\n",
       "4  0.857303  0.001885  0.040230  0.038345"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# KS值对应的阈值\n",
    "a['TPR-FPR'] = a['命中率'] - a['假警报率']\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4744656418256471"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 另外一种获取KS值的方式\n",
    "max(a['TPR-FPR'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>阈值</th>\n",
       "      <th>假警报率</th>\n",
       "      <th>命中率</th>\n",
       "      <th>TPR-FPR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>0.27769</td>\n",
       "      <td>0.255419</td>\n",
       "      <td>0.729885</td>\n",
       "      <td>0.474466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          阈值      假警报率       命中率   TPR-FPR\n",
       "224  0.27769  0.255419  0.729885  0.474466"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 获取KS值对应的阈值等信息\n",
    "a[a['TPR-FPR'] == max(a['TPR-FPR'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
